#define CUDA
#include <iostream>
#include <hip/hip_runtime.h>
#include <hipblas.h>
#include <mpi.h>
#include "gptl.h"
#include<sys/time.h>
#include <chrono>
//#include "inc/matrixpro.hpp"

#define TIMERSTART(tag)  auto tag##_start = std::chrono::steady_clock::now()
#define TIMEREND(tag)  auto tag##_end =  std::chrono::steady_clock::now()
#define DURATION_s(tag) printf("%s costs %d s\n",#tag,std::chrono::duration_cast<std::chrono::seconds>(tag##_end - tag##_start).count())
#define DURATION_ms(tag) \
   int c1,c2,numprocs; \
c1=std::chrono::duration_cast<std::chrono::milliseconds>(tag##_end - tag##_start).count();\
        MPI_Allreduce(&c1,&c2,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);\
        MPI_Comm_size(MPI_COMM_WORLD,&numprocs);\
if(mytid[0]==0)printf("%s costs %d ms\n",#tag,c2/numprocs);
//
//printf("%s costs %d ms\n",#tag,std::chrono::duration_cast<std::chrono::milliseconds>(tag##_end - tag##_start).count());

extern "C" void module_halo_mp_glob_updatehalo_real_3d_(float* h_p, int *,int *); 
extern "C" void glob_updatehalo_p_(float *h_p, int *m, int *iter_max,int *its, int *ite, int *kts, int *kte, int *jts, int *jte){
     module_halo_mp_glob_updatehalo_real_3d_(h_p,m,iter_max);
        }
//#define CA
#ifdef CA
extern "C" void  ca_psolve_main(float ep, float* a0, float* f0, float* cm_helm,int iter_max, double *x0, 
    int dep, int  jdep, int  ids, int  ide, int  jds, int  jde, int  kds, int  kde, 
    int ims, int  ime, int  jms, int  jme, int  kms, int  kme, 
    int its, int  ite, int  jts, int  jte, int  kts, int  kte, int *mytid);

extern "C" void  cuca_psolve_main(float ep, float* a0, float* f0, float* cm_helm,int iter_max, double *x0, 
    int dep, int  jdep, int  ids, int  ide, int  jds, int  jde, int  kds, int  kde, 
    int ims, int  ime, int  jms, int  jme, int  kms, int  kme, 
    int its, int  ite, int  jts, int  jte, int  kts, int  kte, int *mytid);
#else
extern "C" void  cu_psolve_main(float ep, float* a0, float* f0, float* cm_helm,int iter_max, double *x0, 
    int dep, int  jdep, int  ids, int  ide, int  jds, int  jde, int  kds, int  kde, 
    int ims, int  ime, int  jms, int  jme, int  kms, int  kme, 
    int its, int  ite, int  jts, int  jte, int  kts, int  kte, int *mytid);
extern "C" void  psolve_main(float ep, float* a0, float* f0, float* cm_helm,int iter_max, double *x0, 
    int dep, int  jdep, int  ids, int  ide, int  jds, int  jde, int  kds, int  kde, 
    int ims, int  ime, int  jms, int  jme, int  kms, int  kme, 
    int its, int  ite, int  jts, int  jte, int  kts, int  kte, int *mytid);
#endif
extern "C" void cu_solve_helmholts_(float* a_helm, float* b_helm, float* cm_helm, float *threshold, double *pi,  
        int *idep, int *jdep,
        int *ids, int *ide, int *jds, int *jde, int *kds, int *kde, int *
        ims, int *ime, int *jms, int *jme, int *kms, int *kme, int *
        its, int *ite, int *jts, int *jte, int *kts, int *kte, int *mytid){


        int iter_max,max_iteration;
 
        iter_max= 5;
        //max_iteration=20;
        //5,10,20; 10,10,17
        //5,10,4; 5,5,8; 10,10,4;
 
        float ep = 1e-11;
 
static int  init_flag=0;
        #ifdef CUDA
        if ( init_flag==0) {
            int deviceCount;
            hipSetDevice(mytid[0]%4);
        hipblasHandle_t h;
        hipblasCreate(&h);
        hipblasSetPointerMode(h, HIPBLAS_POINTER_MODE_DEVICE);
    	hipDeviceSynchronize(); //test gptl time
        }
        #endif	
 
        init_flag=0;

        GPTLsetoption (GPTLoverhead, 0); // Don't print overhead estimate
        GPTLinitialize ();               // Initialize GPTL
        GPTLstart ("gcr_main");  
if(mytid[0]==0) printf("begin psolve_main\n");
        #ifdef CUDA

        cu_psolve_main( ep,a_helm , b_helm, cm_helm,iter_max, pi,
                *idep, *jdep,  *ids, *ide, *jds, *jde, *kds, *kde,
                *ims, *ime, *jms, *jme, *kms, *kme, 
                *its, *ite, *jts, *jte, *kts, *kte, mytid);
       
        #else


        psolve_main( ep,a_helm , b_helm, cm_helm,iter_max, pi,
                *idep, *jdep,  *ids, *ide, *jds, *jde, *kds, *kde,
                *ims, *ime, *jms, *jme, *kms, *kme, 
                *its, *ite, *jts, *jte, *kts, *kte, mytid);

        #endif
if(mytid[0]==0) printf("end psolve_main\n");
        GPTLstop ("gcr_main");               // Stop the manual timer
        #ifdef CA
GPTLpr_summary_file(MPI_COMM_WORLD,"outfile_ca");
#else
GPTLpr_summary_file(MPI_COMM_WORLD,"outfile_cu");
        #endif
//        if(mytid[0]==0)GPTLpr(mytid[0]);
        GPTLfinalize();
        
  //  DURATION_ms(gcr_main);
        /*
       gettimeofday(&tv_end,NULL);
    
        printf("tv_begin_sec:%d\n",tv_begin.tv_sec);

        printf("tv_begin_usec:%d\n",tv_begin.tv_usec);

        printf("tv_end_sec:%d\n",tv_end.tv_sec);

        printf("tv_end_usec:%d\n",tv_end.tv_usec);
        */
//#endif
/*    
		#ifdef CUDA
#ifdef cagcr
        GPTLpr_file ("outfile_cuca");         // Write output to "outfile"
#else
        GPTLpr_file ("outfile_cu");         // Write output to "outfile"
#endif
		#else
#ifdef cagcr
        GPTLpr_file ("outfile_ca");         // Write output to "outfile"
#else
        GPTLpr_file ("outfile");         // Write output to "outfile"
#endif
		#endif
 */
        init_flag=1;
}

